package com.jiuzhou.crawlers.taobao.pipeline.taobao;

import java.io.PrintWriter;
import java.util.Map;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.log4j.Logger;

import com.jiuzhou.crawlers.taobao.processor.taobao.ItemDetailTaoBaoProcessor;
import com.jiuzhou.crawlers.taobao.util.HdfsUtil;

import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;
	/**
	 * 项目名称：大数据爬虫
	 * @author yanganzhen
	 * version-code：1.0
     * 类说明：淘宝商品详情持久化类
     * 创建时间：2016年12月7日
	 */
public class TaobaoItemDetailPipeline implements Pipeline{
    private String filePath;//hadoop文件夹路径
    public static StringBuffer goodsStringBuffer = new StringBuffer();//商品信息缓存字符串
    private static  org.apache.log4j.Logger log = Logger.getLogger(TaobaoItemDetailPipeline.class);
    
	/** 爬虫持久化类构造函数，传入文件路径
	 * @param filePath
	 */
	public TaobaoItemDetailPipeline(String filePath){
		this.filePath =filePath;
	}
    
	@Override
	public synchronized  void process(ResultItems resultItems, Task arg1) {
		if (!this.filePath.endsWith("/"))
			this.filePath += "/";
        
		//输出page.putFiel中的所有键值对
		for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
			goodsStringBuffer.append(entry.getKey() + ":\t" + entry.getValue() + "\n");
		}
		ItemDetailTaoBaoProcessor.flag++;
		//为存储的文件取名，名字为搜索的关键词
		String url = resultItems.getRequest().getUrl().toString();
		String fileName = url.substring(url.indexOf("#")+1,url.length());
        int  fileLineLength = ItemDetailTaoBaoProcessor.brandLineCount.get(fileName).size();

        System.out.println("完成链接提取---"+url +"     flag="+ItemDetailTaoBaoProcessor.flag);
        	//if(ItemDetailTaoBaoPreocessor.flag == ItemDetailTaoBaoPreocessor.brandLineCount.get(fileName).get(ItemDetailTaoBaoPreocessor.line)){
             if(ItemDetailTaoBaoProcessor.flag ==5000){
        		//String fileType = fileName.substring(fileName.indexOf("+")+1,fileName.length());
    			String path = this.filePath+"/detail/"+DigestUtils.md5Hex(resultItems.getRequest().getUrl());
        		HdfsUtil.WriteToHdfs(path,goodsStringBuffer.toString());
        	  	goodsStringBuffer.delete(0,goodsStringBuffer.length()-1);
        	  	System.out.println("完成爬取，文件名为"+path);
        	  	ItemDetailTaoBaoProcessor.flag=0;   
        }
        
        
/*        	if(ItemDetailTaoBaoPreocessor.flag == ItemDetailTaoBaoPreocessor.brandLineCount.get(fileName).get(ItemDetailTaoBaoPreocessor.line)){
        		String fileType = fileName.substring(fileName.indexOf("+")+1,fileName.length());
    			String path = this.filePath+fileType+"/detail"+fileName+"/line"+ItemDetailTaoBaoPreocessor.line;
        		HdfsUtil.WriteToHdfs(path,goodsStringBuffer.toString());
        	  	goodsStringBuffer.delete(0,goodsStringBuffer.length()-1);
        	  	System.out.println("完成爬取，文件名为"+path);
         	  	if(ItemDetailTaoBaoPreocessor.line ==fileLineLength){
        	  		ItemDetailTaoBaoPreocessor.line=0;
        	  	}
        	  	ItemDetailTaoBaoPreocessor.flag=0;
        		ItemDetailTaoBaoPreocessor.line =ItemDetailTaoBaoPreocessor.line +1 ;
        	}*/
    	  
        	
        }
		
	}

